# -*- coding: utf-8 -*-
'''
Created on Mar 16, 2012

@author: Forever Together
'''
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import datetime
from CrawlerLib.http import Http
import commonlib
from urllib import urlretrieve
from lxml import etree
import simplejson as json
import gzip

DOWNLOAD_PATH = '/detective-conan/'

def download(url, path, filename):
    if (path[-1] != '/'): path += '/'
    downloadPath = path+filename
    print '----- download: {0} to {1}'.format(url, downloadPath)
    if not os.path.isdir(path):
        os.makedirs(path, 0777)
    a, b = urlretrieve(url, downloadPath)
    print a, b

def getListChapterLink(url):
    print 'start process url {0}'.format(url)
    tree = Http.getXMLTree(url)
    if tree == None: 
        print 'Can not build tree'
        return
    data = []
    for item in tree.xpath("//div[@id='content-main']//a"):
        name = commonlib.getElementText(item)
        link = commonlib.getAttribText(item, 'href')
        if link == '': continue
        link = commonlib.urlJoin(url, link)
        images = processChapter(link, name)
        data.append({'chapter': name, 'images': images})
    f = gzip.open('trutien.gz', 'wb')
    jdata = json.dumps(data)
    f.write(jdata)
    f.close()
    

def processChapter(url, chapterName):
    print 'process chapter: {0}'.format(url)
    chapter = commonlib.extractWithRegEx(" (\d+)$", chapterName, 1)
    print chapter
    html = Http.getHtml(url)
    imagesList = commonlib.extractWithRegEx("var (slides2=.+?);", html, 1)
    slides2 = []
    exec(imagesList)
    images = []
    for img in slides2:
        imageLink = commonlib.urlJoin(url, img)
        if imageLink != '':
            imageLink = commonlib.urlJoin(url, imageLink)
            images.append(imageLink)
    images = sorted(images, key=lambda x : x)
    return images;
        
#        imageName = commonlib.extractWithRegEx("/.+-(\d+\.(jpg|JPG|png|PNG|gif|GIF))", imageLink, 1).lower()
#        download(imageLink, "{0}{1}".format(DOWNLOAD_PATH, chapter), imageName)

if __name__ == '__main__':

    getListChapterLink('http://truyentranhtuan.com/tru-tien/')
    
    print 'crawler truyentranhtuan finished at {0}'.format(datetime.datetime.now())
    os._exit(1)
    